#include "../process.h"
#include "../util.h"

#define BATCH 200000

int main(int argc, const char* argv[])
{
	fstream log(argv[1], fstream::in);
	fstream out(argv[2], fstream::out);
	int count = 0;
	cout << "start\n";
	string cur_query, prev_query;
	int query_freq = 0;
	bool first_line = true;
	map<string, int> click_freq;
	map<string, int>::iterator click_freq_itr;
	while(!log.eof())
	{
		string line;
		getline(log, line);
		if(line.length() == 0) continue;
		if(count++%BATCH == 0)cout << ".";
		vector<string> v = util::tokenize(line, "\t");
		cur_query = v[1];
		if(first_line)
		{
			prev_query = cur_query;
			first_line = false;
		}
		if(cur_query != prev_query)
		{
			out << query_freq << "\t" << prev_query;
			for(click_freq_itr = click_freq.begin(); click_freq_itr != click_freq.end(); ++ click_freq_itr)
				out << "\t" << (*click_freq_itr).second << ":" << (*click_freq_itr).first;
			out << endl;
			click_freq.clear();
			query_freq = 0;
			prev_query = cur_query;
		}
		query_freq ++;
		if(v.size() == 5)
		{
			string url = v[4];
			click_freq_itr = click_freq.find(url);
			if(click_freq_itr == click_freq.end())
				click_freq.insert(pair<string, int>(url, 1));
			else
				(*click_freq_itr).second ++;
		}
	}
	out << query_freq << "\t" << prev_query;
	for(click_freq_itr = click_freq.begin(); click_freq_itr != click_freq.end(); ++ click_freq_itr)
		out << "\t" << (*click_freq_itr).second << ":" << (*click_freq_itr).first;
	out << endl;
	click_freq.clear();
	query_freq = 0;
	prev_query = cur_query;
	cout << endl <<count << "\tlines processed!\n";
	log.close();
	out.close();
	return 0;
}